from scrapy import Request, Spider
from urllib import request
import datetime
import requests


def beautify_str(result):
    return result.replace('\n', '').replace('\r', '').replace('\xa0', '').replace("'", '’').strip()


def get_real_link(link):
    header = requests.head(link).headers
    real_link = header['location']
    return real_link


class BaiduHomePageSpider(Spider):
    name = 'baidu_search'
    runtime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    def start_requests(self):
        with open('keywords.txt', encoding='utf-8') as f:
            keywords = f.read().split('\n')
        for keyword in keywords:
            yield Request(
                url='https://www.baidu.com/s?wd={}'.format(request.quote(keyword)),
                callback=self.parse,
                meta={'keyword': keyword}
            )

    def parse(self, response):
        keyword = response.meta['keyword']
        print(keyword)
        items = response.xpath('//div[contains(@class,"c-container")]')
        for item in items:
            title = beautify_str(''.join(item.xpath('.//h3//text()').extract()))
            url = item.xpath('.//h3/a/@href').extract()[0]
            if 'http://www.baidu.com' not in url:
                url = 'http://www.baidu.com' + url
            if '最新相关信息' in title:
                detail_items = item.xpath('.//div[@class="c-offset"]/div')
                for item in detail_items:
                    title = beautify_str(''.join(item.xpath('./a//text()').extract()))
                    url = item.xpath('./a/@href').extract()[0]
                    if 'http://www.baidu.com' not in url:
                        url = 'http://www.baidu.com' + url
                    yield {
                        'keyword': keyword,
                        'runtime': self.runtime,
                        'title': title,
                        'url': get_real_link(url),
                        'type': '最新相关'
                    }
            else:
                yield {
                    'keyword': keyword,
                    'runtime': self.runtime,
                    'title': title,
                    'url': get_real_link(url),
                    'type': '首页搜索'
                }
